v

import requests
from lxml import etree
from fake_useragent import UserAgent
from http import cookiejar
import json
from datetime import datetime
import time
import os
import re
from html import unescape

requests.packages.urllib3.disable_warnings()  # 忽略HTTPS安全警告

"""
今日头条-app
搜索内容
https://lf-hl.snssdk.com/api/search/content/
正文内容
https://a3.pstatp.com/article/full/lite/16/1
评论内容
https://lf-hl.snssdk.com/article/v4/tab_comments/
"""

class TouTiao():
    def __init__(self):
        #声明一个CookieJar对象实例来保存cookie
        self.cookie = cookiejar.CookieJar()
        ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
        self.headers = {
            "Accept": "text/javascript, text/html, application/xml, text/xml, */*",
            "X-Requested-With": "XMLHttpRequest",
            "User-Agent": "Mozilla/5.0 (Linux; Android 5.0.2; CHE-TL00H Build/HonorCHE-TL00H) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/37.0.0.0 Mobile Safari/537.36 JsSdk/2 NewsArticle/7.2.6 NetType/wifi (NewsLite 7.2.6)",
            "Content-Type": "application/x-www-form-urlencoded",
            "Accept-Encoding": "gzip,deflate",
            "Accept-Language": "zh-CN,en-US;q=0.8",
            "Cookie": "odin_tt=afa47b4167a231b742e21892d09a077777a240478b9727338e9c40f5828b5eb4445a81c2d4a2ce1ce309e27ca2dc7302f41aec6288d3fcef721fdc77821139ca; install_id=96695177596; ttreq=1$d4723a8c72b589e7c4174f4e5de19f23ce2618ad",
        }

    def get_contents(self, offset):
        postData = {
            # "search_id":"",
            # "from_search_subtab":"",
            "source":"input",
            # "has_count":"",
            "offset":offset,
            "is_ttwebview":"0",
            "count":"10",
            "action_type":"input_keyword_search",
            # "keyword_type":"",
            "from":"news",  # 频道:"news"资讯 "search_tab" 综合
            "search_position":"news",  # 频道:"news"咨询 "search_bar"综合
            "format":"json",
            "is_from_native":"1",
            "keyword":"曹德旺",
            "pd":"information",
            "qc_query":"",
            "iid":"96695177596",
            # "device_id":"66937151804",
            # "ac":"wifi",
            "channel":"lite2_wandoujia",
            "aid":"35",
            "app_name":"news_article_lite",
            # "version_code":"726",
            # "version_name":"7.2.6",
            "device_platform":"android",
            "ab_version":"668906%2C1355638%2C1367335%2C668903%2C668905%2C668907%2C808414%2C772539%2C1210943%2C668908%2C668904%2C1350383%2C914859%2C928942%2C1322037%2C1335194%2C1355499%2C1355400",
            "ab_client":"a1%2Cc2%2Ce1%2Cf2%2Cg2%2Cf7",
            "ab_feature":"z1",
            "abflag":"3",
            "ssmix":"a",
            "device_type":"CHE-TL00H",
            "device_brand":"Honor",
            "language":"zh",
            "os_api":"21",
            "os_version":"5.0.2",
            "uuid":"869139022485730",
            "openudid":"a2bd3dc65bdf35cd",
            "manifest_version_code":"726",
            # "resolution":"720*1280",
            "dpi":"320",
            "update_version_code":"72603",
            "_rticket":int(time.time()*1000),
            "plugin_state":"64184347",
            "sa_enable":"0",
            # "rom_version":"emotionui_3.1_che-tl00hc00b260-wanmei",
            "cdid":"f5fd06c7-953d-44ec-ac60-44fe5a24b36e"
        }
        url = "https://lf-hl.snssdk.com/api/search/content/"
        time.sleep(3)
        html = HttpUtils.do_request("GET", url, self.headers, postData)

        data_json = json.loads(html.text)
        for d in data_json['data']:
            dict_data = dict()
            id = d['id']
            dict_data['abstract'] = d['abstract']  # 摘要
            dict_data['title'] = d['title']  # 标题
            dict_data['media_name'] = d['media_name']  # 媒体名称
            dict_data['source'] = d['source']  # 作者
            dict_data['source_url'] = d['source_url']  # 文章链接
            dict_data['create_time'] = datetime.fromtimestamp(int(d['create_time']))\
                .strftime("%Y-%m-%d %H:%M:%S")  # 发布时间
            dict_data['comment_count'] = d['comment_count']  # 评论数量
            dict_data['digg_count'] = d['digg_count']  # 点赞数量
            dict_data['read_count'] = d['read_count']  # 点赞数量
            if d.get('user_auth_info') is not None:
                dict_data['auth_type'] = d['user_auth_info']['auth_type']  # 0 是V账号
            print(f"============={dict_data['title']}=============")
            # 判断日期范围
            date_start = datetime.strptime('2020-01-01', "%Y-%m-%d")
            date_end = datetime.strptime('2020-04-30', "%Y-%m-%d")
            date_now = datetime.strptime(dict_data['create_time'].split(" ")[0], "%Y-%m-%d")
            if date_now >= date_start and date_now <= date_end:
                # 正文内容
                self.get_contents_1(id, dict_data)
            # 评论内容
            self.get_comments(id, 0)

    # 正文内容
    def get_contents_1(self, id, dict_data):
        postData = {
            "iid":"96695177596",
            "device_id":"66937151804",
            "ac":"wifi",
            "channel":"lite2_wandoujia",
            "aid":"35",
            "app_name":"news_article_lite",
            "version_code":"726",
            "version_name":"7.2.6",
            "device_platform":"android",
            "ab_version":"668905%2C668907%2C808414%2C772539%2C1210943%2C668908%2C668904%2C668906%2C1355638%2C1367333%2C668903%2C914859%2C928942%2C1322037%2C1335194%2C1355499%2C1355400",
            "ab_client":"a1%2Cc4%2Ce1%2Cf2%2Cg2%2Cf7",
            "ab_feature":"z1",
            "abflag":"3",
            "ssmix":"a",
            "device_type":"CHE-TL00H",
            "device_brand":"Honor",
            "language":"zh",
            "os_api":"21",
            "os_version":"5.0.2",
            "uuid":"869139022485730",
            "openudid":"a2bd3dc65bdf35cd",
            "manifest_version_code":"726",
            "resolution":"720*1280",
            "dpi":"320",
            "update_version_code":"72603",
            "_rticket":"1577083263440",
            "plugin_state":"64184347",
            "sa_enable":"0",
            "rom_version":"emotionui_3.1_che-tl00hc00b260-wanmei",
            "cdid":"f5fd06c7-953d-44ec-ac60-44fe5a24b36e"
            # 'iid':'2154687474445303',
            # 'device_id':'66937151804',
            # 'ac':'wifi',
            # 'mac_address':'48%3Adb%3A50%3Ab9%3A85%3A30',
            # 'channel':'baidu_0411',
            # 'aid':'13',
            # 'app_name':'news_article',
            # 'version_code':'774',
            # 'version_name':'7.7.4',
            # 'device_platform':'android',
            # 'ab_version':'1419049%2C668775%2C1529247%2C1190522%2C1157750%2C1419597%2C1629530%2C1469498%2C1484966%2C1576657%2C1593455%2C1742591%2C668779%2C662099%2C1403340%2C1706992%2C668774%2C1702947%2C660830%2C662176%2C1738571',
            # 'ab_feature':'102749%2C94563',
            # 'ssmix':'a',
            # 'device_type':'CHE-TL00H',
            # 'device_brand':'Honor',
            # 'language':'zh',
            # 'os_api':'21',
            # 'os_version':'5.0.2',
            # 'uuid':'869139022485730',
            # 'openudid':'a2bd3dc65bdf35cd',
            # 'manifest_version_code':'7740',
            # 'resolution':'720*1280',
            # 'dpi':'320',
            # 'update_version_code':'77410',
            # '_rticket':'1590975723396',
            # 'tma_jssdk_version':'1.65.0.5',
            # 'rom_version':'emotionui_3.1_che-tl00hc00b260-wanmei',
            # 'plugin':'0',
            # 'host_abi':'armeabi-v7a',
            # 'cdid':'8a761eee-dc56-4b3c-b1d6-117df15b910b'
        }
        url = f"https://a3-ipv6.pstatp.com/article/content/24/1/{id}/{id}/0/0/1"
        headers = {
            "Accept-Encoding": "gzip",
            "User-Agent": "Dalvik/2.1.0 (Linux; U; Android 5.0.2; CHE-TL00H Build/HonorCHE-TL00H) NewsArticle/7.2.6 ttnet okhttp/3.10.0.2",
            "X-Gorgon": "0404b8b80001f6f8c90fd67a320552abd1b63e987e59493e0669",
            "X-Khronos": "1590975723",
            "Host": "a3-ipv6.pstatp.com",
            "Connection": "Keep-Alive",
            # 'Host': 'a3-ipv6.pstatp.com',
            # 'Connection': 'keep-alive',
            # 'X-SS-DP': '13',
            # 'x-tt-trace-id': '00-6d8af91409f95c3e13cb4d6736c3000d-6d8af91409f95c3e-01',
            # 'User-Agent': 'com.ss.android.article.news/7740 (Linux; U; Android 5.0.2; zh_CN; CHE-TL00H; Build/HonorCHE-TL00H; Cronet/TTNetVersion:e4fc96b0 2020-03-30 QuicVersion:0144d358 2020-03-24)',
            # 'Accept-Encoding': 'gzip, deflate',
            # 'X-Gorgon': '0404b8b80001f6f8c90fd67a320552abd1b63e987e59493e0669',
            # 'X-Khronos': '1590975723'

        }
        time.sleep(3)
        html = HttpUtils.do_request("GET", url, headers, postData)
        data_json = json.loads(html.text)
        # dict_data = dict()
        dict_data['content'] = data_json['data']['content']  # 内容
        with open("996.txt", "a+", encoding="utf-8") as fp:
            fp.write("--------------------正文内容--------------------" + '\n')

            fp.write("标题:" + dict_data['title'] + '\n')
            fp.write("作者:" + dict_data['source'] + '\n')
            fp.write("发布时间:" + dict_data['create_time'] + '\n')
            fp.write("评论数量:" + str(dict_data['comment_count']) + '\n')
            fp.write("正文内容:" + self.html_to_plain_text(dict_data['content']) + '\n')
        print("=============获取正文=============")
        # 评论内容
        self.get_comments(id, 0)


    # 评论内容
    def get_comments(self, id, pagenum):
        postData = {
            "count":"20",
            "category":"",
            "comment_request_from":"1",
            "group_id": id,  # "6769757758414651908",
            "offset":pagenum,
            "fold":"1",
            "iid":"96695177596",
            "device_id":"66937151804",
            "ac":"wifi",
            "channel":"lite2_wandoujia",
            "aid":"35",
            "app_name":"news_article_lite",
            "version_code":"726",
            "version_name":"7.2.6",
            "device_platform":"android",
            "ab_version":"668905%2C668907%2C808414%2C772539%2C1210943%2C668908%2C668904%2C668906%2C1355638%2C1367335%2C668903%2C914859%2C928942%2C1322037%2C1335194%2C1355499%2C1355400",
            "ab_client":"a1%2Cc2%2Ce1%2Cf2%2Cg2%2Cf7",
            "ab_feature":"z1",
            "abflag":"3",
            "ssmix":"a",
            "device_type":"CHE-TL00H",
            "device_brand":"Honor",
            "language":"zh",
            "os_api":"21",
            "os_version":"5.0.2",
            "uuid":"869139022485730",
            "openudid":"a2bd3dc65bdf35cd",
            "manifest_version_code":"726",
            "resolution":"720*1280",
            "dpi":"320",
            "update_version_code":"72603",
            "_rticket":int(time.time()*1000),
            "plugin_state":"64184347",
            "sa_enable":"0",
            "rom_version":"emotionui_3.1_che-tl00hc00b260-wanmei",
            "cdid":"f5fd06c7-953d-44ec-ac60-44fe5a24b36e"
        }
        url = f"https://lf-hl.snssdk.com/article/v4/tab_comments/"
        html = HttpUtils.do_request("GET", url, self.headers, postData)
        time.sleep(3)
        if html.text != "":
            data_json = json.loads(html.text)
            counts = int(data_json['total_number'])
            if counts > 0:
                for d in data_json['data']:
                    dict_data = dict()
                    dict_data['user_name'] = d['comment']['user_name']  # 用户名称
                    dict_data['content'] = d['comment']['text']  # 评论内容
                    dict_data['digg_count'] = str(d['comment']['digg_count'])  # 点赞数量
                    dict_data['create_time'] = datetime.fromtimestamp(int(d['comment']['create_time'])) \
                        .strftime("%Y-%m-%d %H:%M:%S")  # 发布时间
                    with open("996.txt", "a+", encoding="utf-8") as fp:
                        fp.write("--------------------评论内容--------------------" + '\n')
                        fp.write("用户名称:" + dict_data['user_name'] + '\n')
                        fp.write("点赞数量:" + dict_data['digg_count'] + '\n')
                        fp.write("评论时间:" + dict_data['create_time'] + '\n')
                        fp.write("评论内容:" + dict_data['content'] + '\n')
                    print(f"=============获取【{dict_data['user_name']}】评论=============")
                if pagenum <= counts:
                    pagenum += 20
                    time.sleep(5)
                    self.get_comments(id, pagenum)

    # 删除HTML不需要标签
    def html_to_plain_text(self, html):
        text = re.sub('<head.*?>.*?</head>', '', html, flags=re.M | re.S | re.I)
        text = re.sub('<a\s.*?>', ' HYPERLINK ', text, flags=re.M | re.S | re.I)
        text = re.sub('<.*?>', '', text, flags=re.M | re.S)
        # 过滤标签，保留标签p
        # text = re.sub('(?!<(p).*?>|<(/p).*?>)<.*?>', '', text, flags=re.M | re.S)
        # text = re.sub(r'(\s*\n)+', '\n', text, flags=re.M | re.S)
        text = text.replace("\n", "")
        return unescape(text)

if __name__ == '__main__':
    toutiao = TouTiao()
    for p in range(0, 300, 10):
        toutiao.get_contents(p)
        print(f"=======第{p}条=========")
        time.sleep(3)
    # toutiao.get_comments("6679961468382741000", 0)
    # toutiao.get_contents_1("6772468852472152590", "")